# Scrapy settings for asos project
#
# For simplicity, this file contains only the most important settings by
# default. All the other settings are documented here:
#
#     http://doc.scrapy.org/topics/settings.html
#
# Or you can copy and paste them from where they're defined in Scrapy:
# 
#     scrapy/conf/default_settings.py
#

BOT_NAME = 'asos'
BOT_VERSION = '1.0'

SPIDER_MODULES = ['asos.spiders']
NEWSPIDER_MODULE = 'asos.spiders'
DEFAULT_ITEM_CLASS = 'asos.items.AsosItem'
USER_AGENT = '%s/%s' % (BOT_NAME, BOT_VERSION)

ITEM_PIPELINES = ['asos.pipelines.CSVPipeline',
                  'asos.pipelines.JsonPipeline']

LOG_FILE = 'log.log'
LOG_LEVEL = 'DEBUG'

RETRY_TIMES = 10
CONCURRENT_REQUESTS_PER_SPIDER=10
DOWNLOAD_DELAY = 0.25
DOWNLOAD_TIMEOUT = 1800
